# -*- coding: utf-8 -*-
import re

import time
import urllib2

from lxml import etree
import scrapy
from scrapy.http import Request
from bs4 import BeautifulSoup
from dingdian.items import NsfocusItem, SecurityFocusItem
import sys
import pymysql

from dingdian.init_mysql import session

from dingdian.models import VulNsfocus, VulSecurityFocus
from dingdian.schedule import schedule

reload(sys)
sys.setdefaultencoding( "utf-8" )

class Myspider(scrapy.Spider):

    name = 'securityFocus'
    allowedDomains = 'https://www.securityfocus.com'
    bash_url = 'https://www.securityfocus.com/cgi-bin/index.cgi?o=0000&l=100&c=12&op=display_list&vendor=&version=&title=&CVE='

    pageNumPattern = re.compile(r'\b\d+?(?=\))')
    currentPageNumPattern = re.compile(r'\d+')
    cvePattern = re.compile(r'CVE-\d{4}-\d+')
    headers = {
        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
        'accept-Language': 'zh-CN,zh;q=0.9',
        'accept-encoding':'gzip,deflate,br',
        'cache-control': 'no-cache',
        'pragma': 'no-cache',
        'connection': 'keep-alive',
        'host': 'www.securityfocus.com',
        'referer': 'www.securityfocus.com/vulnerabilities',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36',
        'cookie':'__utmz=33565859.1523522903.1.1.utmccn=(referral)|utmcsr=google.com|utmcct=/|utmcmd=referral; __utma=33565859.82979994.1523522903.1523522903.1525192918.2; __utmc=33565859; __utmb=33565859; sfses='
	}

    pageNum = 0
    vulNum = 0
    totalVuls = 0
    currentVuls = 0.0

    def start_requests(self):
        url = self.bash_url
        yield Request(url,headers=self.headers,callback= self.parse)

    def parse(self, response):
        #print(response.text)

        pageNumSpan = BeautifulSoup(response.text, 'lxml').find('span', class_='pages').get_text()

        self.pageNum = int(re.search(self.pageNumPattern, pageNumSpan).group())

        for num in range(1,self.pageNum+1):
            url = self.allowedDomains + '/cgi-bin/index.cgi?o=%s&l=100&c=12&op=display_list&vendor=&version=&title=&CVE=' % str((num-1)*100)
            yield Request(url, callback=self.get_vul,headers=self.headers)

    def get_vul(self, response):
        pageNumSpan = BeautifulSoup(response.text, 'lxml').find('span', class_='pages').get_text()

        currentPageNum = int(re.findall(self.currentPageNumPattern, pageNumSpan)[0])
        #print currentPageNum
        hrefs = BeautifulSoup(response.text, 'html.parser').find('div',attrs={ 'style':r'padding: 4px;'}).find_all('a')
        self.vulNum=len(hrefs)/2
        self.totalVuls=self.pageNum*self.vulNum
        for n in range(0, self.vulNum):
        #for n in range(0, 2):
            #   print(hrefs[n]['href'])
            url = self.allowedDomains + hrefs[2*n]['href']
            if (session.query(VulSecurityFocus).filter(VulSecurityFocus.url == url).all()) != []:
                print "已经解析过该网页"
                self.currentVuls += 1.0
                schedule(self.totalVuls, self.currentVuls)
                continue
            # print(url)
            yield Request(url, callback=self.save_vul,headers=self.headers)

    def save_vul(self, response):
        self.currentVuls += 1.0
        schedule(self.totalVuls, self.currentVuls)
        pageSource = response.text
        vulDetail = BeautifulSoup(pageSource, "lxml").find("div", id="vulnerability")

        item=SecurityFocusItem()
        item['url'] = response.url
        item['title'] = BeautifulSoup(pageSource, 'lxml').find('title').get_text()
        item['cveID'] = re.search(self.cvePattern, vulDetail.get_text()).group() if re.search(self.cvePattern,vulDetail.get_text()) is not None else None
        item['type'] = vulDetail.find('span',text='Class:').next_element.next_element.next_element.next_element.get_text().strip()

        publishTime = vulDetail.find('span',text = 'Published:').next_element.next_element.next_element.next_element.get_text().strip()
        item['publishTime'] = time.strftime("%Y-%m-%d", time.strptime(publishTime, "%b %d %Y %I:%M%p"))

        updateTime = vulDetail.find('span',text='Updated:').next_element.next_element.next_element.next_element.get_text().strip()
        item['updateTime'] = time.strftime("%Y-%m-%d", time.strptime(updateTime, "%b %d %Y %I:%M%p"))
        effectSys =vulDetail.find('span',text='Vulnerable:').next_element.next_element.next_element.next_element.get_text().strip()
        if(len(effectSys)>4096):
            item['effectSys'] = None
        else:
            item['effectSys'] =effectSys.replace('\n\t\t\t\t\t\n\t\t\t\t\n\t\t\t\t\t','\n')
        yield Request(response.url+'/discuss', headers=self.headers,callback=self.getDiscuss,meta={'item':item})


    def getDiscuss(self,response):
        item = response.meta['item']
        item['message']= BeautifulSoup(response.text, "lxml").find("div", id="vulnerability").get_text()
        #print item
        yield Request(item['url'] + '/solution', headers=self.headers,callback=self.getSolution, meta={'item': item})



    def getSolution(self, response):
        item = response.meta['item']
        try:
            item['suggestion'] = BeautifulSoup(response.text, "lxml").find("div", id="vulnerability").find('b',text="Solution:").next_element.next_element.next_element
        except:
            item['suggestion'] = None
        #print item
        yield Request(item['url'] + '/references', headers=self.headers,callback=self.getReferences, meta={'item': item})

    def getReferences(self,response):
        item = response.meta['item']
        vendorPatches = BeautifulSoup(response.text, "lxml").find("div", id="vulnerability").find_all('a')

        vendorPatch = ''
        for i in range(len(vendorPatches)):
            vendorPatch = vendorPatches[i]["href"] + "\n"
        item["vendorPatch"] = vendorPatch
        item['chinese'] = False
        if item['type'] == 'Unknown':
            item['type'] = None
        item['total'] = self.totalVuls
        item['current'] = int(self.currentVuls)
        yield item

